import json, os, sys
from tqdm import tqdm


def load_json(path):
    with open(path) as json_file:
        data = json.load(json_file)
        return data


def read_file(path):
    tmp = os.popen("wc -l {}".format(path)).readline()
    line_count = int(tmp.split()[0])
    pbar = tqdm(total=line_count)

    corpus = []
    # CQA = []
    idx = 0
    with open(path, 'r') as file:
        pbar.set_description(path)
        pbar.refresh()
        tmp = []
        for line in file:
            line = line.strip()
            if line == '' and len(tmp) != 0:
                # idx += 1
                # print(idx)
                try:
                    # CQA.append({'content': tmp[:-2], 'question': tmp[-2], 'answer': tmp[-1]})
                    corpus.append(' '.join([' '.join(s) for s in tmp[:-2]]))
                    tmp = []
                except:
                    print('error')
                    pass
            else:
                # tmp.append(stopwords(line.strip().split()))
                tmp.append(line.strip().split())
            pbar.update(1)
        if len(tmp):
            # idx += 1
            # print(idx)
            # CQA.append({'content': tmp[:-2], 'question': tmp[-2], 'answer': tmp[-1]})
            corpus.append([' '.join(s) for s in tmp[:-2]])
    return corpus


# 得到纯字符集
def stopwords(sent):
    # pattern = re.compile(u"[\u4e00-\u9fa5]+")
    new_sent = []
    for word in sent:
        if not word in stopword:
            new_sent.append(word)
    return new_sent


def write2file(str, path):
    with open(path, 'a') as file:
        file.writelines(str)
        file.writelines('\n')


if __name__ == '__main__':
    # stopword = load_json('data/stopwords.json')
    corpus = read_file('raw/train.txt')
    for c in corpus:
        write2file(c, 'out/corpus_no_stopwords.txt')
    # for cqa in CQA:
    #     write2file(json.dumps(cqa, ensure_ascii=False), 'out/cqa.txt')
